
(defun skimList (list) "takes out all useless words from the list"
  (setq templist *ignorelist*)
  (while (!= templist nil) do
    (setq list (remove (car templist) list))
    (setq templist (cdr templist))
  )
  list
)




(defun wordInventory (wordlist) "creates a list for each unique word and counts the frequency, and creates a list of words named inventory and a list of frequencies named frequencies"
  (setq words '())
  (setq frequencies '())
  (setq tempwordlist wordlist)
  (setq temp nil)

  (while (!= tempwordlist nil) do

      ;;create info for a word if it does not already exist
      (if (not (isIncluded (car tempwordlist) words))
	  (progn
	  (setq words (list words (car tempwordlist)))
	  (setq frequencies (list frequencies 0))
	  (setq words (flatten words))
	  (setq frequencies (flatten frequencies))
	  )
      )


      ;;add 1 to frequency if another copy is found
      (if (isIncluded (car tempwordlist) words)
	  (progn
	  (setq temp (wordindex (car tempwordlist) words))
	  (setf (nth temp frequencies) (+ 1 (nth temp frequencies)))
	  )

      )

  (setq tempwordlist (cdr tempwordlist))

  )
  (setq inventory (list words frequencies))
  inventory

)


(defun isVerbList (wordlist) "creates a list of 1's or 0's (true or false) for whether a given word is a verb or not, currently not in use"
 (setq verblist (MakeList "VerbList"))
 (setq tempwordlist wordlist)
 (setq addon nil)
 (setq isVerbList '())

 (while (!= tempwordlist nil) do
    (if (isIncluded (car tempwordlist) verblist)
        ;;then
	(setq addon 1)

        ;;else
        (setq addon 0)
    )

;;if a word ends in ize, its most likely a verb, we need to check for that

    (if (equal (type-of (car tempwordlist)) 'symbol) ;;make sure its a symbol

	(progn

	(setq symstring (symbol-name (car tempwordlist)))

        (if (equal "IZE" (last-n-char 3 symstring))
	    (setq addon 1)
	)

	(if (equal "ISE" (last-n-char 3 symstring))
	    (setq addon 1)
	)
   
	;;different forms of the verb
	(if (equal "IZES" (last-n-char 4 symstring))
	    (setq addon 1)
	)

	(if (equal "ISES" (last-n-char 4 symstring))
	    (setq addon 1)
	)

	(if (equal "IZED" (last-n-char 4 symstring))
	    (setq addon 1)
	)

	(if (equal "ISED" (last-n-char 4 symstring))
	    (setq addon 1)
	)
	
	)
    )
 (setq isVerbList (list isVerbList addon))
 (setq isVerbList (flatten isVerbList))
 (setq tempwordlist (cdr tempwordlist))
 )
isVerbList
)





(defun paragraph-split (file) "takes an email file and splits it into a list of lists, each nested list contains a paragraph"
 
 (setq my-file (open file))
 (setq tempstring "")
 (setq paragraphlist '())
 (setq character t)
 (while (!= character nil) do
	(setq character (read-char my-file nil))


	;;keep adding
	(setq tempstring (concatenate 'string tempstring (string character)))

	;;if there there is a new paragraph
	(if (or (equal character #\Newline) (equal character #\Tab)) 
	    (progn
	    (setq tempstring (removePunc tempstring))
	    (setq tempstring (add-paren-to-string tempstring))
	    (setq paragraphlist (cons (read-from-string tempstring) paragraphlist))
	    (setq tempstring "")
	    )
	)

 )
 (remove nil (reverse paragraphlist))

)


(defun sentence-split (file) "takes and email file and splits it into a list of lists, each nested list contains paragraph"
  
 (setq my-file (open file))
 (setq tempstring "")
 (setq sentencelist '())
 (setq character t)
 (while (!= character nil) do
	(setq character (read-char my-file nil))


	;;keep adding
	(setq tempstring (concatenate 'string tempstring (string character)))

	;;if there there is a new paragraph
	(if (equal character #\.)
	    (progn
	    (setq tempstring (removePunc tempstring))
	    (setq tempstring (add-paren-to-string tempstring))
	    (setq sentencelist (cons (read-from-string tempstring) sentencelist))
	    (setq tempstring "")
	    )
	)

 )
 (remove nil (reverse sentencelist))

)